import requests
import time
import random
from bs4 import BeautifulSoup

base_url = 'https://www.pythontab.com/html/pythonhexinbiancheng/'
header_value = {
    'User-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.39',
}
proxies = [
    {'http': 'http://121.232.148.167:9000'},
    {'http': 'http://39.105.28.28:8118'},
    {'http': 'http://113.195.18.133:9999'},
]

for i in range(1, 28):
    if i > 1:
        url = base_url + str(i) + ".html"
    else:
        url = base_url
    print(url)
    try:
        proxy_index = random.randint(0, 2)
        r = requests.get(url, headers=header_value, proxies=proxies[proxy_index])
        soup = BeautifulSoup(r.text, 'lxml')
        titles = soup.select('#catlist  a')
        for title in titles:
            print(title.text)
    except:
        print("请求失败")
    else:
        print(r.status_code)
        print(r.request.headers)

    sleep_time = random.randint(0, 2) + random.random()
    time.sleep(sleep_time)
